import scrapy
import json
from ..items import *
from copy import deepcopy
import math
import time


class ListSpider(scrapy.Spider):
	name = 'list'
	# allowed_domains = ['xx.com']
	start_urls = ['http://www.gdzwfw.gov.cn/portal/item-solr/getCommonAuditItem']

	headers = {
		'Accept': 'application/json, text/javascript, */*; q=0.01',
		'Host': 'www.gdzwfw.gov.cn',
		'Referer': 'http://www.gdzwfw.gov.cn/portal/personal/theme?region=440000&catalogCode=',
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
		'X-Requested-With': 'XMLHttpRequest'
	}
	url_com = 'http://www.gdzwfw.gov.cn/portal/guide/{}'

	flag = True
	next_flag = True

	next_dict = {}

	def start_requests(self):
		from_data = {
			'pageNum': '1',
			'pageSize': '10',
			'TASK_TYPE': '',
			'DEPT_CODE': '',
			'AREA_CODE': '440300',
			'ISLOCALLEVEL': '0',
			'KEY_WORD': '',
			'TYPE': '',
			'IS_ONLINE': '',
			'TASKTAG': '1',
			'LIMIT_SCENE_SUM': '0',
			'IS_ZERO_SECNE': ''
		}
		yield scrapy.FormRequest(
			method="POST",
			headers=self.headers,
			callback=self.parse,
			formdata=from_data,
			url=self.start_urls[0],
			meta={
				'from_data': deepcopy(from_data),
				'Item_types': '个人事项'
			}
		)

		from_data = {
			'pageNum': '1',
			'pageSize': '10',
			'TASK_TYPE': '',
			'DEPT_CODE': '',
			'AREA_CODE': '440300',
			'ISLOCALLEVEL': '0',
			'KEY_WORD': '',
			'TYPE': '',
			'IS_ONLINE': '',
			'TASKTAG': '2',
			'LIMIT_SCENE_SUM': '0',
			'IS_ZERO_SECNE': ''
		}
		yield scrapy.FormRequest(
			method="POST",
			headers=self.headers,
			callback=self.parse,
			formdata=from_data,
			url=self.start_urls[0],
			meta={
				'from_data': deepcopy(from_data),
				'Item_types': '法人事项'
			}
		)

	def parse(self, response, **kwargs):
		Item_types = response.meta['Item_types']
		from_data = response.meta['from_data']
		json_data = json.loads(response.text).get('data')
		if json_data:
			json_data_dict = json_data.get('CUSTOM')
			# 具体数据
			json_data_list = json_data_dict.get('AUDIT_ITEMLIST')
			item = DataList()
			if json_data_list:
				for data in json_data_list:
					item['AREACODE'] = data.get('AREACODE')
					item['AREA_CODE_TEXT'] = data.get('AREA_CODE_TEXT')
					item['CATAID'] = data.get('CATAID')
					item['CATANAME'] = data.get('CATANAME')
					item['DEPT_NAME'] = data.get('DEPT_NAME')
					item['LIMIT_SCENE_NUM'] = data.get('LIMIT_SCENE_NUM')
					item['ONLINECHECK'] = data.get('ONLINECHECK')
					item['ORGCODE'] = data.get('ORGCODE')
					item['TASK_CODE'] = data.get('TASK_CODE')
					item['TASK_NAME'] = data.get('TASK_NAME')
					item['TASK_STATE'] = data.get('TASK_STATE')
					item['TASK_TAG'] = data.get('TASK_TAG')
					item['TASK_TYPE'] = data.get('TASK_TYPE')
					item['TASK_TYPE_TEXT'] = data.get('TASK_TYPE_TEXT')
					item['URL'] = data.get('URL')
					item['parentCatalogId'] = data.get('parentCatalogId')
					item['parentCatalogName'] = data.get('parentCatalogName')
					item['subCatalogName'] = data.get('subCatalogName')
					item['subTaskName'] = data.get('subTaskName')
					item['Item_types'] = Item_types
					item['Guide_to'] = self.url_com.format(data.get('TASK_CODE'))
					# 初始化为空
					item['Update_tag'] = ""

					# 暂时已 Guide_to  和 Item_types
					res = self.is_down('data_list', 'Guide_to', item['Guide_to'], "Item_types", item['Item_types'])
					if not res:
						yield item

			# 总页数
			if self.next_dict.get(f"next{Item_types}", 1):
				self.next_dict[f"next{Item_types}"] = False
				PAGEINFO = json_data_dict.get('PAGEINFO')
				TOTALNUM = PAGEINFO.get('TOTALNUM')
				PAGESIZE = PAGEINFO.get('PAGESIZE')
				# # 总页数
				page_num = math.ceil(TOTALNUM / PAGESIZE)
				print(f'进入翻页操作{page_num}')
				for i in range(2, page_num + 1):
					time.sleep(0.5)
					from_data['pageNum'] = str(i)
					yield scrapy.FormRequest(
						method="POST",
						headers=self.headers,
						callback=self.parse,
						formdata=from_data,
						url=self.start_urls[0],
						meta={
							'from_data': deepcopy(from_data),
							'Item_types':Item_types
						}
					)
		else:
			print('无数据', from_data)

	def is_down(self, base, ziduan, code, ziduan_2, code_2):
		sql = """
	        select id 
	        from {} where {} = '{}' and {} = '{}'
	    """.format(base, ziduan, code, ziduan_2, code_2)
		self.cursor.execute(sql)
		result = self.cursor.fetchone()
		return result
